#!/usr/bin/env python
# Parse Apache access logs 
import re
import os
import sys
import time
import cPickle as pickle
from gzip import GzipFile

# 62.10.167.194 - - [03/Feb/2009:12:32:50 -0800] "GET /imageupload/169b91eb4f40b9bed92068818efcb9f4/Foto578.jpg HTTP/1.1" 200 341477 "http://forumgamesradar.futuregamer.it/showthread.php?t=710908" "Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.9.0.5) Gecko/2008120122 Firefox/3.0.5"

DB = os.path.join(os.environ['HOME'],'apachedb.pickle')
ERROR = 'error.log'

reline = re.compile('^[\d.]+ - - \[(\d{2})/(\w{3})/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([-+]\d{2})\d{2}\] "(.+?) (.+?) .+?"')
# ('03', 'Feb', '2009', '12', '32', '50', '-08', 'GET',
# '/imageupload/169b91eb4f40b9bed92068818efcb9f4/Foto578.jpg')

quiet = True

def main():
    try:
        f = file(DB)
        db = pickle.load(f)
        f.close()
    except IOError:
        print 'Create db...'
        db = {}

    for fname in sys.argv[1:]:
        print 'File',fname
        if fname.endswith('.gz'):
            f = GzipFile(fname)
        else:
            f = file(fname)
        for line in f:
            match = reline.match(line)
            if match:
                line = match.groups()
                if line[7] == 'GET':
                    db.setdefault(line[8],set())
                    d = build_date(*line[:7]) 
                    db[line[8]].add(d)
                    if not quiet:
                        print d,line[8]
        f.close()

    f = file(DB,'w')
    pickle.dump(db,f)
    f.close()

def error(s):
    f = file(ERROR,'a')
    f.write(s+'\n')
    f.close()

def build_date(d,mo,y,h,mi,s,o):

    if o[0] == '-':
        o = int(o[1:])
    elif o[0] == '+':
        o = -int(o[1:])
    else:
        assert 0

    mounths = {
        'Jan':1,
        'Feb':2
        }
    
    #return '%s-%.2d-%s' % (y,mounths[m],d)
    return int(time.mktime((int(y),mounths[mo],int(d),int(h)+o,int(mi),int(s),0,0,0)))

if __name__=="__main__":
    main()
